{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from dataset_io import *\n",
    "from classifiers import *\n",
    "from algorithm import Algorithm, Predictions\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#np.save(\"cross_validated_scenes\",cross_validated_scenes())\n",
    "cross_validated_scenes = np.load(\"cross_validated_scenes.npy\", allow_pickle=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Classifiers evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate_classifiers():\n",
    "    grad_metrics=[]\n",
    "    color_metrics=[]\n",
    "\n",
    "    for fold in tqdm(cross_validated_scenes):\n",
    "        train_scenes, test_scenes, fold_name = fold\n",
    "        print(fold_name)\n",
    "        acc, prec, rec, f1, _ = classify_gradient(train_scenes, test_scenes, fold_name, True)\n",
    "        grad_metrics.append([acc, prec, rec, f1])\n",
    "        res = classify_colors(train_scenes, test_scenes, fold_name, True)\n",
    "        color_metrics.append(res)\n",
    "\n",
    "    np.save(\"color_metrics\", color_metrics)\n",
    "    np.save(\"grad_metrics\", grad_metrics)\n",
    "    \n",
    "#evaluate_classifiers()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "color_metrics_avg\n",
      "[[0.85106473 0.76691145 0.81843552 0.79046445]\n",
      " [0.89788648 0.83362693 0.8696812  0.85032493]\n",
      " [0.88802002 0.79463581 0.90192808 0.84422685]]\n",
      "grad_metrics_avg\n",
      "[0.98093372 0.9983972  0.92963724 0.95946163]\n"
     ]
    }
   ],
   "source": [
    "color_metrics = np.load(\"color_metrics.npy\")\n",
    "grad_metrics = np.load(\"grad_metrics.npy\")\n",
    "grad_metrics_avg = np.mean(grad_metrics, 0)\n",
    "color_metrics_avg = np.mean(color_metrics, 0)\n",
    "print(\"color_metrics_avg\")\n",
    "print(color_metrics_avg)\n",
    "print(\"grad_metrics_avg\")\n",
    "print(grad_metrics_avg)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Algorithms evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/471 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fold_0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 471/471 [19:15<00:00,  2.45s/it]\n",
      "  0%|          | 2/471 [00:00<00:37, 12.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.454972486252307 0.04673774881042295 12325599 12325599 0.08981628797604502 0.10267146950033552\n",
      "draw_and_calc on  ['max_norm', 'min_norm', 'sum_norm', 'weighted_sum', 'weighted_sum_norm']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 471/471 [00:37<00:00, 12.68it/s]\n",
      "100%|██████████| 471/471 [00:38<00:00, 12.31it/s]\n",
      "100%|██████████| 471/471 [00:38<00:00, 12.23it/s]\n",
      "100%|██████████| 471/471 [00:38<00:00, 12.21it/s]\n",
      "100%|██████████| 471/471 [00:38<00:00, 12.19it/s]\n",
      "  0%|          | 0/577 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "agg_overlaps [0.707206624077748, 0.7386761118203433, 0.7923524286260464, 0.7743143210336396, 0.7784286028107406]\n",
      "fold_1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 577/577 [31:50<00:00,  3.31s/it]\n",
      "  2%|▏         | 14/577 [00:00<00:04, 135.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4668496453354856 0.053456405468358295 18619464 18619464 0.08873977213305832 0.10233067168182765\n",
      "draw_and_calc on  ['max_norm', 'min_norm', 'sum_norm', 'weighted_sum', 'weighted_sum_norm']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 577/577 [00:16<00:00, 34.86it/s] \n",
      "100%|██████████| 577/577 [00:17<00:00, 33.60it/s] \n",
      "100%|██████████| 577/577 [00:17<00:00, 33.35it/s] \n",
      "100%|██████████| 577/577 [00:17<00:00, 33.51it/s] \n",
      "100%|██████████| 577/577 [00:17<00:00, 33.38it/s] \n",
      "  0%|          | 0/494 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "agg_overlaps [0.5456379925121185, 0.5351914002435159, 0.6468301139190661, 0.6321994342616164, 0.633096536193109]\n",
      "fold_2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 492/494 [23:31<00:04,  2.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No more predictions with prob > 0.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 494/494 [23:36<00:00,  2.87s/it]\n",
      "  2%|▏         | 10/494 [00:00<00:05, 94.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4537218004038781 0.03733969858401964 13582674 13582674 0.08738752648854739 0.08538633446805602\n",
      "draw_and_calc on  ['max_norm', 'min_norm', 'sum_norm', 'weighted_sum', 'weighted_sum_norm']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 494/494 [00:15<00:00, 32.18it/s] \n",
      "100%|██████████| 494/494 [00:16<00:00, 30.47it/s] \n",
      "100%|██████████| 494/494 [00:16<00:00, 30.20it/s] \n",
      "100%|██████████| 494/494 [00:16<00:00, 29.85it/s] \n",
      "100%|██████████| 494/494 [00:16<00:00, 30.06it/s] \n",
      "  0%|          | 0/555 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "agg_overlaps [0.49339115355596824, 0.5533990300382999, 0.5754682174983902, 0.5512634589959666, 0.5565702747391826]\n",
      "fold_3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 555/555 [28:37<00:00,  3.09s/it]\n",
      "  0%|          | 2/555 [00:00<00:48, 11.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4621282024793426 0.04723494147321165 14093589 14093589 0.08829061476349989 0.09533258673115663\n",
      "draw_and_calc on  ['max_norm', 'min_norm', 'sum_norm', 'weighted_sum', 'weighted_sum_norm']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 555/555 [00:27<00:00, 20.08it/s]\n",
      "100%|██████████| 555/555 [00:27<00:00, 20.47it/s]\n",
      "100%|██████████| 555/555 [00:26<00:00, 20.91it/s]\n",
      "100%|██████████| 555/555 [00:26<00:00, 20.66it/s]\n",
      "100%|██████████| 555/555 [00:27<00:00, 20.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "agg_overlaps [0.6065061336850093, 0.5031431331952374, 0.6668734142539184, 0.6520711202371184, 0.6550005743346193]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "def evaluate_algorithm():\n",
    "    overlaps=[]\n",
    "    color_means=[]\n",
    "    grad_means=[]\n",
    "    color_counts=[]\n",
    "    grad_counts=[]\n",
    "    color_stds=[]\n",
    "    grad_stds=[]\n",
    "\n",
    "    for fold in cross_validated_scenes:\n",
    "        train_scenes, test_scenes, fold_name = fold\n",
    "        print(fold_name)\n",
    "        alg = Algorithm(train_scenes, test_scenes, fold_name)\n",
    "        res_path = alg.run()\n",
    "        color_mean, grad_mean, color_count, grad_count, color_std, grad_std = alg.get_norm_coeffs()\n",
    "        color_means.append(color_mean)\n",
    "        grad_means.append(grad_mean)\n",
    "        color_counts.append(color_count)\n",
    "        grad_counts.append(grad_count)\n",
    "        color_stds.append(color_std)\n",
    "        grad_stds.append(grad_std)\n",
    "        \n",
    "        print(color_mean, grad_mean, color_count, grad_count, color_std, grad_std)\n",
    "        agg_overlaps = Predictions(res_path, test_scenes).draw_and_calc()\n",
    "        print(\"agg_overlaps\",agg_overlaps)\n",
    "        overlaps.append(agg_overlaps)\n",
    "\n",
    "    color_mean = np.average(color_means, weights=np.divide(color_counts,len(color_counts)))\n",
    "    grad_mean = np.average(grad_means, weights=np.divide(grad_counts,len(grad_counts)))\n",
    "    color_std = np.mean(color_stds)\n",
    "    grad_std = np.mean(grad_stds)\n",
    "    norm_coeffs = [color_mean, grad_mean, color_std, grad_std]\n",
    "    np.save(PROJECT_PATH+\"results/agg_overlaps_iou_nw\", np.mean(overlaps, 0))\n",
    "    np.save(PROJECT_PATH+\"results/norm_coeffs_iou\", norm_coeffs)\n",
    "    \n",
    "evaluate_algorithm()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.58818548, 0.58260242, 0.67038104, 0.65246208, 0.655774  ])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.load(PROJECT_PATH+\"results/agg_overlaps_iou_nw.npy\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.46017551, 0.04681373, 0.08855855, 0.09643027])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.load(PROJECT_PATH+\"results/norm_coeffs_iou.npy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### color, grad, max, min, sum, max_norm, min_norm, sum_norm, weighted_sum, weighted_sum_norm\n",
    "\n",
    "0.49258194, 0.59860443, 0.59728843, 0.68011496, 0.68655563, 0.59936776, 0.62096155, 0.69024978, 0.67709032, 0.68211362\n",
    "\n",
    "full [0.49258194, 0.59860443, 0.59728843, 0.6804365 , 0.68655563,\n",
    "       0.59936776, 0.62096155, 0.69024978, 0.67745126, 0.68211362]\n",
    "       \n",
    "IoU 0.3829, 0.6065, 0.5782, 0.6440, 0.6645, 0.58818548, 0.58260242, 0.67038104, 0.65246208, 0.655774\n",
    "\n",
    "\n",
    "#### color_mean, grad_mean, color_std, grad_std\n",
    "\n",
    "0.46017551, 0.04681373, 0.08855855, 0.09643027"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
